import nltk
from nltk import *
import string
from nltk.corpus import stopwords
from nltk.book import text1

# print(nltk.find("."))

# print(word_tokenize("Hello nltk"))

paragraph = "You must follow me carefully. I shall have to controvert one or two ideas that are almost universally accepted. The geometry, for instance, they taught you at school is founded on a misconception."
tokenized_text = sent_tokenize(paragraph)
# print(tokenized_text)
text = "You must follow me carefully."
res = text.translate(str.maketrans(string.punctuation, " " * len(string.punctuation)))
print(res)
tokenized_word = word_tokenize(text)
print(tokenized_text)
punctuation = list(string.punctuation)
res = [word for word in tokenized_word if word not in punctuation]
print(res)

# 去除停用词
stop_words = stopwords.words("english")
print(stop_words)

text = "I shall have to controvert one or two ideas that are almost universally accepted."
tokenized_word = word_tokenize(text)
tokenized_word = [word for word in tokenized_word if word not in punctuation]
tokenized_word = [word for word in tokenized_word if word not in stop_words]
print(tokenized_word)

# 词频提取
tokenized_word = word_tokenize(paragraph)
tokenized_word = [word for word in tokenized_word if word not in punctuation]
print(tokenized_word)
word_freqs = nltk.FreqDist([w.lower() for w in tokenized_word])
print(word_freqs.items())
#word_freqs.plot()

print(text1.concordance("boy"))